{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "\n",
    "from helper import nn\n",
    "from helper import logistic_regression as lr\n",
    "\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 400)\n",
      "(5000,)\n"
     ]
    }
   ],
   "source": [
    "raw_X, raw_y = nn.load_data('ex3data1.mat')\n",
    "print(raw_X.shape)\n",
    "print(raw_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 401)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add intercept=1 for x0\n",
    "X = np.insert(raw_X, 0, values=np.ones(raw_X.shape[0]), axis=1)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 5000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# y have 10 categories here. 1..10, they represent digit 0 as category 10 because matlab index start at 1\n",
    "# I'll ditit 0, index 0 again\n",
    "y_matrix = []\n",
    "\n",
    "for k in range(1, 11):\n",
    "    y_matrix.append((raw_y == k).astype(int))\n",
    "\n",
    "# last one is k==10, it's digit 0, bring it to the first position\n",
    "y_matrix = [y_matrix[-1]] + y_matrix[:-1]\n",
    "y = np.array(y_matrix)\n",
    "\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train 1 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t0 = lr.logistic_regression(X, y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(401,)\n",
      "Accuracy=0.9974\n"
     ]
    }
   ],
   "source": [
    "print(t0.shape)\n",
    "y_pred = lr.predict(X, t0)\n",
    "print('Accuracy={}'.format(np.mean(y[0] == y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is this real......"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train k model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 401)\n"
     ]
    }
   ],
   "source": [
    "k_theta = np.array([lr.logistic_regression(X, y[k]) for k in range(10)])\n",
    "print(k_theta.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# making prediction\n",
    "* think about the shape of k_theta, now you are making $X\\times\\theta^T$\n",
    "> $(5000, 401) \\times (10, 401).T = (5000, 10)$\n",
    "* after that, you run sigmoid to get probabilities and for each row, you find the highest prob as the answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prob_matrix = lr.sigmoid(X @ k_theta.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.99577632,  0.        ,  0.00053528, ...,  0.00006469,\n",
       "         0.00003912,  0.00172187],\n",
       "       [ 0.99834614,  0.0000001 ,  0.00005607, ...,  0.00009684,\n",
       "         0.0000029 ,  0.00008492],\n",
       "       [ 0.99139772,  0.        ,  0.00056828, ...,  0.00000654,\n",
       "         0.02653902,  0.00197393],\n",
       "       ..., \n",
       "       [ 0.00000068,  0.04140121,  0.00320887, ...,  0.00012723,\n",
       "         0.00297489,  0.70761228],\n",
       "       [ 0.00001843,  0.00000013,  0.00000009, ...,  0.00164768,\n",
       "         0.06814687,  0.86116757],\n",
       "       [ 0.02879673,  0.        ,  0.00012974, ...,  0.36626836,\n",
       "         0.00497918,  0.14821854]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True)\n",
    "prob_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_pred = np.argmax(prob_matrix, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_answer = raw_y.copy()\n",
    "y_answer[y_answer==10] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.99      0.98       500\n",
      "          1       0.95      0.99      0.97       500\n",
      "          2       0.95      0.92      0.93       500\n",
      "          3       0.95      0.91      0.93       500\n",
      "          4       0.95      0.95      0.95       500\n",
      "          5       0.92      0.92      0.92       500\n",
      "          6       0.97      0.98      0.97       500\n",
      "          7       0.95      0.95      0.95       500\n",
      "          8       0.93      0.92      0.92       500\n",
      "          9       0.92      0.92      0.92       500\n",
      "\n",
      "avg / total       0.94      0.94      0.94      5000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_answer, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tf]",
   "language": "python",
   "name": "conda-env-tf-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
